import os
domo_username = os.environ.get("DOMO_USERNAME")
domo_instance = os.environ['DOMO_INSTANCE']
domo_password = os.environ.get("DOMO_PASSWORD")Tutorial - Generate a DomoStats dataset using DomoLibrary
DomoStats
MagicETL
import requests
def get_full_auth(
domo_instance, domo_username, domo_password
) -> str: # returns a session token
"""use username and password to generate an access token"""
url = f"https://{domo_instance}.domo.com/api/content/v2/authentication"
body = {
"method": "password",
"emailAddress": domo_username,
"password": domo_password,
}
res = requests.request(method="POST", url=url, json=body)
data = res.json()
token = data.get("sessionToken")
if not token:
raise Exception("unable to retrieve a session token")
return token
session_token = get_full_auth(domo_instance, domo_username, domo_password)
assert session_tokendef generate_query_dataflows_body(count=10, offset=0):
return {
"entities": ["DATAFLOW"],
"count": count,
"offset": offset,
"combineResults": False,
"query": "*",
"filters": [
{
"filterType": "term",
"field": "data_flow_type",
"value": "MAGIC",
"name": "Magic ETL v2",
"not": False,
}
],
}
body_dataflows = generate_query_dataflows_body(10, 0)
body_dataflows{'entities': ['DATAFLOW'],
'count': 10,
'offset': 0,
'combineResults': False,
'query': '*',
'filters': [{'filterType': 'term',
'field': 'data_flow_type',
'value': 'MAGIC',
'name': 'Magic ETL v2',
'not': False}]}def query_datacenter(
domo_instance,
session_token,
body,
return_raw: bool = False,
debug_api: bool = False,
):
"""function that queries the datacenter and for a specific body"""
url = f"https://{domo_instance}.domo.com/api/search/v1/query"
if debug_api:
print({"url": url, "body": body})
headers = {"x-domo-authentication": session_token}
res = requests.request(url=url, method="POST", headers=headers, json=body)
if return_raw:
return res
data = res.json()
return data.get("searchObjects")
dataflows_ls = query_datacenter(
domo_instance=domo_instance,
session_token=session_token,
body=body_dataflows,
debug_api=True,
return_raw=False,
)
print(len(dataflows_ls))
dataflows_ls[0]{'url': 'https://domo-community.domo.com/api/search/v1/query', 'body': {'entities': ['DATAFLOW'], 'count': 10, 'offset': 0, 'combineResults': False, 'query': '*', 'filters': [{'filterType': 'term', 'field': 'data_flow_type', 'value': 'MAGIC', 'name': 'Magic ETL v2', 'not': False}]}}
10{'entityType': 'dataflow',
'databaseId': '112',
'searchId': {'indexName': None,
'databaseId': '112',
'customer': 'mmmm-0012-0200',
'entityType': 'dataflow'},
'createDate': 1670612239000,
'lastModified': 1671217571000,
'lastIndexed': 1677789510843,
'highlightedFields': {},
'language': 'English',
'requestAccess': False,
'score': 63.42418,
'name': 'Time Zone Test',
'description': '',
'ownedById': '1345408759',
'ownedByName': 'Alexis Lorenz (DataMaven)',
'tags': [],
'lastRunDate': 1671639517000,
'inputDatasets': [{'name': 'Time Zone Test',
'id': '3929c2cd-6549-4f53-9539-0f3e34233ef0'}],
'outputDatasets': [{'name': 'Time Zone Test | Output',
'id': 'df5f6051-6842-460a-995b-ffcd9721b361'}],
'runCount': 14,
'successRate': 1.0,
'dataFlowType': 'MAGIC',
'status': 'SUCCESS',
'deleted': False,
'passwordProtected': False,
'abandoned': False,
'owners': [{'id': '1345408759',
'type': 'USER',
'displayName': 'Alexis Lorenz (DataMaven)'}],
'ownersLocalized': {'localizedMessage': 'Alexis Lorenz (DataMaven)',
'count': 1},
'paused': False,
'statusOrderPriority': 70,
'inputCount': 1,
'outputCount': 1,
'winnerText': 'Time Zone Test',
'ownedByType': 'USER',
'customer': 'mmmm-0012-0200'}dataflow_ids = [dataflow.get("databaseId") for dataflow in dataflows_ls]
dataflow_ids['112', '406', '353', '131', '227', '37', '185', '38', '226', '124']def get_dataflow_definition(dataflow_id, session_token, return_raw: bool = False):
url = f"https://{domo_instance}.domo.com/api/dataprocessing/v1/dataflows/{dataflow_id}"
headers = {"x-domo-authentication": session_token}
res = requests.request(method="get", url=url, headers=headers)
if return_raw:
return res
data = res.json()
return data
get_dataflow_definition(dataflow_ids[0], session_token=session_token){'id': 112,
'name': 'Time Zone Test',
'dapDataFlowId': 'f486ec50-bf91-4201-bd42-627cc94427ff',
'responsibleUserId': 1345408759,
'runState': 'ENABLED',
'onboardFlowVersion': {'id': 460,
'timeStamp': 1671217571000,
'authorId': 1345408759,
'description': 'Updated notes',
'numInputs': 1,
'numOutputs': 1,
'executionCount': 0,
'executionSuccessCount': 0,
'versionNumber': 10},
'lastExecution': {'id': 242749,
'onboardFlowId': 112,
'previewRows': 0,
'dapDataFlowExecutionId': '2b45d20a-5371-492e-82ef-569ced13c51b',
'beginTime': 1671639506000,
'endTime': 1671639517000,
'lastUpdated': 1671639517000,
'failed': False,
'state': 'SUCCESS',
'dataFlowVersion': 460},
'created': 1670612239000,
'modified': 1671217571000,
'actions': [{'type': 'LoadFromVault',
'id': '74e125a8-76fb-49e6-8825-c1d0ee121641',
'name': 'Time Zone Test',
'gui': {'x': 36,
'y': 180,
'color': 3238043,
'colorSource': None,
'sampleJson': None},
'dataSourceId': '3929c2cd-6549-4f53-9539-0f3e34233ef0',
'executeFlowWhenUpdated': True,
'pseudoDataSource': False,
'truncateTextColumns': False,
'truncateRows': False,
'onlyLoadNewVersions': False},
{'type': 'ExpressionEvaluator',
'id': '7a287510-67d9-40e2-9a6e-d8f247b8b005',
'name': 'Offset Prep',
'dependsOn': ['74e125a8-76fb-49e6-8825-c1d0ee121641'],
'notes': [{'x1': None,
'y1': None,
'x2': None,
'y2': None,
'body': 'There are notes embedded in the configuration. '}],
'gui': {'x': 144,
'y': 180,
'color': None,
'colorSource': None,
'sampleJson': None},
'expressions': [{'expression': 'now()',
'fieldName': 'Now (UTC)',
'settings': None},
{'expression': "convert_TZ(now(),'UTC',`Actual Time Zone of Data`)",
'fieldName': 'Now (Local)',
'settings': None},
{'expression': "convert_TZ(now(),'UTC','US/Eastern')",
'fieldName': 'Now (Eastern)',
'settings': None},
{'expression': "convert_TZ(now(),'UTC','US/Central')",
'fieldName': 'Now (Central)',
'settings': None},
{'expression': "convert_TZ(now(),'UTC','US/Mountain')",
'fieldName': 'Now (Mountain)',
'settings': None},
{'expression': "convert_TZ(now(),'UTC','US/Pacific')",
'fieldName': 'Now (Pacific)',
'settings': None},
{'expression': "'Notes about this transform'\n-- Need to establish current time in each target time zone to enable the calculation of the current offset to account for daylight savings. If it is only necessary to see the data in the time zone from the system, only the UTC and the system's time zone are needed. ",
'fieldName': 'Notes',
'settings': None}]},
{'type': 'DateCalculator',
'id': 'be94da12-290e-41bc-a8f2-a076348a9b1a',
'name': 'Date Operations',
'dependsOn': ['7a287510-67d9-40e2-9a6e-d8f247b8b005'],
'notes': [{'x1': None,
'y1': None,
'x2': None,
'y2': None,
'body': "This is the calculation of the current offset to account for daylight savings. \n\nIf it is only necessary to see the data in the time zone from the system, only the system's time zone are needed. "}],
'gui': {'x': 264,
'y': 180,
'color': None,
'colorSource': None,
'sampleJson': None},
'calculations': [{'fieldName': 'UTC offset of Now',
'calcType': 'DATE_DIFF_HR',
'fieldA': 'Now (UTC)',
'constantA': None,
'exprA': None,
'fieldB': 'Now (Local)',
'constantB': None,
'exprB': None,
'fieldC': None,
'constantC': None,
'exprC': None,
'settings': None},
{'fieldName': 'UTC Offset of Eastern',
'calcType': 'DATE_DIFF_HR',
'fieldA': 'Now (UTC)',
'constantA': None,
'exprA': None,
'fieldB': 'Now (Eastern)',
'constantB': None,
'exprB': None,
'fieldC': None,
'constantC': None,
'exprC': None,
'settings': None},
{'fieldName': 'UTC Offset of Central',
'calcType': 'DATE_DIFF_HR',
'fieldA': 'Now (UTC)',
'constantA': None,
'exprA': None,
'fieldB': 'Now (Central)',
'constantB': None,
'exprB': None,
'fieldC': None,
'constantC': None,
'exprC': None,
'settings': None},
{'fieldName': 'UTC Offset of Mountain',
'calcType': 'DATE_DIFF_HR',
'fieldA': 'Now (UTC)',
'constantA': None,
'exprA': None,
'fieldB': 'Now (Mountain)',
'constantB': None,
'exprB': None,
'fieldC': None,
'constantC': None,
'exprC': None,
'settings': None},
{'fieldName': 'UTC Offset of Pacific',
'calcType': 'DATE_DIFF_HR',
'fieldA': 'Now (UTC)',
'constantA': None,
'exprA': None,
'fieldB': 'Now (Pacific)',
'constantB': None,
'exprB': None,
'fieldC': None,
'constantC': None,
'exprC': None,
'settings': None}]},
{'type': 'ExpressionEvaluator',
'id': '65e20743-2ca9-4bd7-ac1d-7b4a27ee9c5c',
'name': 'Time Conversions 1',
'dependsOn': ['be94da12-290e-41bc-a8f2-a076348a9b1a'],
'notes': [{'x1': None,
'y1': None,
'x2': None,
'y2': None,
'body': 'There are notes embedded in the configuration. '}],
'gui': {'x': 384,
'y': 180,
'color': None,
'colorSource': None,
'sampleJson': None},
'expressions': [{'expression': '`DateTime (Local - Domo Assumes UTC)`\n-- Wanted column name in webform to be descriptive, but this is clearer for output.',
'fieldName': 'Local Time',
'settings': None},
{'expression': "CONVERT_TZ(`DateTime (Local - Domo Assumes UTC)`,`Actual Time Zone of Data`,'UTC')\n-- Converts the date/time from the local time to UTC so that conversion to each time zone can be made. ",
'fieldName': 'UTC',
'settings': None},
{'expression': "'Notes on CONVERT_TZ transforms'\n-- These transforms will reflect the data's timestamp in each of the desired time zones. Time Zone can be used in a variable so end-users can select in which time zone to show the data. If it is only necessary to see the data in the time zone from the system, only the UTC and the system's time zone are needed. ",
'fieldName': 'Notes',
'settings': None},
{'expression': "CONVERT_TZ(`UTC`,'UTC','US/Eastern')",
'fieldName': 'US/Eastern',
'settings': None},
{'expression': "CONVERT_TZ(`UTC`,'UTC','US/Central')",
'fieldName': 'US/Central',
'settings': None},
{'expression': "CONVERT_TZ(`UTC`,'UTC','US/Mountain')",
'fieldName': 'US/Mountain',
'settings': None},
{'expression': "CONVERT_TZ(`UTC`,'UTC','US/Pacific')",
'fieldName': 'US/Pacific',
'settings': None},
{'expression': "'Notes on Adjustments'\n-- This is where we account for the instantaneous gap between UTC and the time zone of the data. If, at this moment, it's a different day UTC than in the target time zone, this will shift all the data forward a day. This is the field to be used in the relative date filters. Data cards remain the same. Where time zone selection is desired, variables should be used. ",
'fieldName': 'Notes',
'settings': None},
{'expression': 'case when hour(now()) <= `UTC offset of Now` \n then Date(DATE_ADD(`Local Time`,interval 1 day))\nelse date(`Local Time`) end',
'fieldName': 'Local Time - Adjusted UTC Date',
'settings': None},
{'expression': 'case when hour(now()) <= `UTC Offset of Eastern` \n then Date(DATE_ADD(`US/Eastern`,interval 1 day))\nelse date(`US/Eastern`) end',
'fieldName': 'US/Eastern - Adjusted UTC Date',
'settings': None},
{'expression': 'case when hour(now()) <= `UTC Offset of Central` \n then Date(DATE_ADD(`US/Central`,interval 1 day))\nelse date(`US/Central`) end',
'fieldName': 'US/Central - Adjusted UTC Date',
'settings': None},
{'expression': 'case when hour(now()) <= `UTC Offset of Mountain`\n then Date(DATE_ADD(`US/Mountain`,interval 1 day))\nelse date(`US/Mountain`) end',
'fieldName': 'US/Mountain - Adjusted UTC Date',
'settings': None},
{'expression': 'case when hour(now()) <= `UTC Offset of Pacific`\n then Date(DATE_ADD(`US/Pacific`,interval 1 day))\nelse date(`US/Pacific`) end',
'fieldName': 'US/Pacific - Adjusted UTC Date',
'settings': None}]},
{'type': 'PublishToVault',
'id': '3e7dd3a3-9f6a-4a66-a709-4e9f2172bb36',
'name': 'Time Zone Test | Output',
'dependsOn': ['65e20743-2ca9-4bd7-ac1d-7b4a27ee9c5c'],
'gui': {'x': 504,
'y': 180,
'color': None,
'colorSource': None,
'sampleJson': None},
'dataSource': {'guid': 'df5f6051-6842-460a-995b-ffcd9721b361',
'type': 'DataFlow',
'name': 'Time Zone Test | Output',
'cloudId': 'domo'},
'versionChainType': 'REPLACE',
'schemaSource': 'DATAFLOW',
'partitioned': False}],
'engineProperties': {'kettle.mode': 'STRICT'},
'inputs': [{'dataSourceId': '3929c2cd-6549-4f53-9539-0f3e34233ef0',
'executeFlowWhenUpdated': True,
'dataSourceName': 'Time Zone Test',
'onlyLoadNewVersions': False}],
'outputs': [{'onboardFlowId': None,
'dataSourceId': 'df5f6051-6842-460a-995b-ffcd9721b361',
'dataSourceName': 'Time Zone Test | Output',
'versionChainType': 'REPLACE'}],
'executionCount': 14,
'executionSuccessCount': 14,
'hydrationState': 'DEHYDRATED',
'useLegacyTriggerBehavior': False,
'passwordProtected': False,
'deleted': False,
'abandoned': False,
'neverAbandon': False,
'settings': {},
'triggerSettings': {'triggers': [{'title': 'Dataset trigger',
'triggerEvents': [{'datasetId': '3929c2cd-6549-4f53-9539-0f3e34233ef0',
'triggerOnDataChanged': True,
'type': 'DATASET_UPDATED'}],
'triggerConditions': [],
'triggerId': None}],
'zoneId': 'UTC',
'locale': 'en_US'},
'paused': False,
'enabled': True,
'restricted': False,
'container': False,
'databaseType': 'MAGIC',
'triggeredByInput': True,
'draft': False,
'editable': True,
'numInputs': 1,
'numOutputs': 1,
'magic': True,
'subsetProcessing': False}dataflow_definition_ls = [
get_dataflow_definition(dataflow_id, session_token=session_token)
for dataflow_id in dataflow_ids
]
dataflow_definition_ls[0]{'id': 112,
'name': 'Time Zone Test',
'dapDataFlowId': 'f486ec50-bf91-4201-bd42-627cc94427ff',
'responsibleUserId': 1345408759,
'runState': 'ENABLED',
'onboardFlowVersion': {'id': 460,
'timeStamp': 1671217571000,
'authorId': 1345408759,
'description': 'Updated notes',
'numInputs': 1,
'numOutputs': 1,
'executionCount': 0,
'executionSuccessCount': 0,
'versionNumber': 10},
'lastExecution': {'id': 242749,
'onboardFlowId': 112,
'previewRows': 0,
'dapDataFlowExecutionId': '2b45d20a-5371-492e-82ef-569ced13c51b',
'beginTime': 1671639506000,
'endTime': 1671639517000,
'lastUpdated': 1671639517000,
'failed': False,
'state': 'SUCCESS',
'dataFlowVersion': 460},
'created': 1670612239000,
'modified': 1671217571000,
'actions': [{'type': 'LoadFromVault',
'id': '74e125a8-76fb-49e6-8825-c1d0ee121641',
'name': 'Time Zone Test',
'gui': {'x': 36,
'y': 180,
'color': 3238043,
'colorSource': None,
'sampleJson': None},
'dataSourceId': '3929c2cd-6549-4f53-9539-0f3e34233ef0',
'executeFlowWhenUpdated': True,
'pseudoDataSource': False,
'truncateTextColumns': False,
'truncateRows': False,
'onlyLoadNewVersions': False},
{'type': 'ExpressionEvaluator',
'id': '7a287510-67d9-40e2-9a6e-d8f247b8b005',
'name': 'Offset Prep',
'dependsOn': ['74e125a8-76fb-49e6-8825-c1d0ee121641'],
'notes': [{'x1': None,
'y1': None,
'x2': None,
'y2': None,
'body': 'There are notes embedded in the configuration. '}],
'gui': {'x': 144,
'y': 180,
'color': None,
'colorSource': None,
'sampleJson': None},
'expressions': [{'expression': 'now()',
'fieldName': 'Now (UTC)',
'settings': None},
{'expression': "convert_TZ(now(),'UTC',`Actual Time Zone of Data`)",
'fieldName': 'Now (Local)',
'settings': None},
{'expression': "convert_TZ(now(),'UTC','US/Eastern')",
'fieldName': 'Now (Eastern)',
'settings': None},
{'expression': "convert_TZ(now(),'UTC','US/Central')",
'fieldName': 'Now (Central)',
'settings': None},
{'expression': "convert_TZ(now(),'UTC','US/Mountain')",
'fieldName': 'Now (Mountain)',
'settings': None},
{'expression': "convert_TZ(now(),'UTC','US/Pacific')",
'fieldName': 'Now (Pacific)',
'settings': None},
{'expression': "'Notes about this transform'\n-- Need to establish current time in each target time zone to enable the calculation of the current offset to account for daylight savings. If it is only necessary to see the data in the time zone from the system, only the UTC and the system's time zone are needed. ",
'fieldName': 'Notes',
'settings': None}]},
{'type': 'DateCalculator',
'id': 'be94da12-290e-41bc-a8f2-a076348a9b1a',
'name': 'Date Operations',
'dependsOn': ['7a287510-67d9-40e2-9a6e-d8f247b8b005'],
'notes': [{'x1': None,
'y1': None,
'x2': None,
'y2': None,
'body': "This is the calculation of the current offset to account for daylight savings. \n\nIf it is only necessary to see the data in the time zone from the system, only the system's time zone are needed. "}],
'gui': {'x': 264,
'y': 180,
'color': None,
'colorSource': None,
'sampleJson': None},
'calculations': [{'fieldName': 'UTC offset of Now',
'calcType': 'DATE_DIFF_HR',
'fieldA': 'Now (UTC)',
'constantA': None,
'exprA': None,
'fieldB': 'Now (Local)',
'constantB': None,
'exprB': None,
'fieldC': None,
'constantC': None,
'exprC': None,
'settings': None},
{'fieldName': 'UTC Offset of Eastern',
'calcType': 'DATE_DIFF_HR',
'fieldA': 'Now (UTC)',
'constantA': None,
'exprA': None,
'fieldB': 'Now (Eastern)',
'constantB': None,
'exprB': None,
'fieldC': None,
'constantC': None,
'exprC': None,
'settings': None},
{'fieldName': 'UTC Offset of Central',
'calcType': 'DATE_DIFF_HR',
'fieldA': 'Now (UTC)',
'constantA': None,
'exprA': None,
'fieldB': 'Now (Central)',
'constantB': None,
'exprB': None,
'fieldC': None,
'constantC': None,
'exprC': None,
'settings': None},
{'fieldName': 'UTC Offset of Mountain',
'calcType': 'DATE_DIFF_HR',
'fieldA': 'Now (UTC)',
'constantA': None,
'exprA': None,
'fieldB': 'Now (Mountain)',
'constantB': None,
'exprB': None,
'fieldC': None,
'constantC': None,
'exprC': None,
'settings': None},
{'fieldName': 'UTC Offset of Pacific',
'calcType': 'DATE_DIFF_HR',
'fieldA': 'Now (UTC)',
'constantA': None,
'exprA': None,
'fieldB': 'Now (Pacific)',
'constantB': None,
'exprB': None,
'fieldC': None,
'constantC': None,
'exprC': None,
'settings': None}]},
{'type': 'ExpressionEvaluator',
'id': '65e20743-2ca9-4bd7-ac1d-7b4a27ee9c5c',
'name': 'Time Conversions 1',
'dependsOn': ['be94da12-290e-41bc-a8f2-a076348a9b1a'],
'notes': [{'x1': None,
'y1': None,
'x2': None,
'y2': None,
'body': 'There are notes embedded in the configuration. '}],
'gui': {'x': 384,
'y': 180,
'color': None,
'colorSource': None,
'sampleJson': None},
'expressions': [{'expression': '`DateTime (Local - Domo Assumes UTC)`\n-- Wanted column name in webform to be descriptive, but this is clearer for output.',
'fieldName': 'Local Time',
'settings': None},
{'expression': "CONVERT_TZ(`DateTime (Local - Domo Assumes UTC)`,`Actual Time Zone of Data`,'UTC')\n-- Converts the date/time from the local time to UTC so that conversion to each time zone can be made. ",
'fieldName': 'UTC',
'settings': None},
{'expression': "'Notes on CONVERT_TZ transforms'\n-- These transforms will reflect the data's timestamp in each of the desired time zones. Time Zone can be used in a variable so end-users can select in which time zone to show the data. If it is only necessary to see the data in the time zone from the system, only the UTC and the system's time zone are needed. ",
'fieldName': 'Notes',
'settings': None},
{'expression': "CONVERT_TZ(`UTC`,'UTC','US/Eastern')",
'fieldName': 'US/Eastern',
'settings': None},
{'expression': "CONVERT_TZ(`UTC`,'UTC','US/Central')",
'fieldName': 'US/Central',
'settings': None},
{'expression': "CONVERT_TZ(`UTC`,'UTC','US/Mountain')",
'fieldName': 'US/Mountain',
'settings': None},
{'expression': "CONVERT_TZ(`UTC`,'UTC','US/Pacific')",
'fieldName': 'US/Pacific',
'settings': None},
{'expression': "'Notes on Adjustments'\n-- This is where we account for the instantaneous gap between UTC and the time zone of the data. If, at this moment, it's a different day UTC than in the target time zone, this will shift all the data forward a day. This is the field to be used in the relative date filters. Data cards remain the same. Where time zone selection is desired, variables should be used. ",
'fieldName': 'Notes',
'settings': None},
{'expression': 'case when hour(now()) <= `UTC offset of Now` \n then Date(DATE_ADD(`Local Time`,interval 1 day))\nelse date(`Local Time`) end',
'fieldName': 'Local Time - Adjusted UTC Date',
'settings': None},
{'expression': 'case when hour(now()) <= `UTC Offset of Eastern` \n then Date(DATE_ADD(`US/Eastern`,interval 1 day))\nelse date(`US/Eastern`) end',
'fieldName': 'US/Eastern - Adjusted UTC Date',
'settings': None},
{'expression': 'case when hour(now()) <= `UTC Offset of Central` \n then Date(DATE_ADD(`US/Central`,interval 1 day))\nelse date(`US/Central`) end',
'fieldName': 'US/Central - Adjusted UTC Date',
'settings': None},
{'expression': 'case when hour(now()) <= `UTC Offset of Mountain`\n then Date(DATE_ADD(`US/Mountain`,interval 1 day))\nelse date(`US/Mountain`) end',
'fieldName': 'US/Mountain - Adjusted UTC Date',
'settings': None},
{'expression': 'case when hour(now()) <= `UTC Offset of Pacific`\n then Date(DATE_ADD(`US/Pacific`,interval 1 day))\nelse date(`US/Pacific`) end',
'fieldName': 'US/Pacific - Adjusted UTC Date',
'settings': None}]},
{'type': 'PublishToVault',
'id': '3e7dd3a3-9f6a-4a66-a709-4e9f2172bb36',
'name': 'Time Zone Test | Output',
'dependsOn': ['65e20743-2ca9-4bd7-ac1d-7b4a27ee9c5c'],
'gui': {'x': 504,
'y': 180,
'color': None,
'colorSource': None,
'sampleJson': None},
'dataSource': {'guid': 'df5f6051-6842-460a-995b-ffcd9721b361',
'type': 'DataFlow',
'name': 'Time Zone Test | Output',
'cloudId': 'domo'},
'versionChainType': 'REPLACE',
'schemaSource': 'DATAFLOW',
'partitioned': False}],
'engineProperties': {'kettle.mode': 'STRICT'},
'inputs': [{'dataSourceId': '3929c2cd-6549-4f53-9539-0f3e34233ef0',
'executeFlowWhenUpdated': True,
'dataSourceName': 'Time Zone Test',
'onlyLoadNewVersions': False}],
'outputs': [{'onboardFlowId': None,
'dataSourceId': 'df5f6051-6842-460a-995b-ffcd9721b361',
'dataSourceName': 'Time Zone Test | Output',
'versionChainType': 'REPLACE'}],
'executionCount': 14,
'executionSuccessCount': 14,
'hydrationState': 'DEHYDRATED',
'useLegacyTriggerBehavior': False,
'passwordProtected': False,
'deleted': False,
'abandoned': False,
'neverAbandon': False,
'settings': {},
'triggerSettings': {'triggers': [{'title': 'Dataset trigger',
'triggerEvents': [{'datasetId': '3929c2cd-6549-4f53-9539-0f3e34233ef0',
'triggerOnDataChanged': True,
'type': 'DATASET_UPDATED'}],
'triggerConditions': [],
'triggerId': None}],
'zoneId': 'UTC',
'locale': 'en_US'},
'paused': False,
'enabled': True,
'magic': True,
'restricted': False,
'subsetProcessing': False,
'container': False,
'databaseType': 'MAGIC',
'triggeredByInput': True,
'draft': False,
'editable': True,
'numInputs': 1,
'numOutputs': 1}def get_action_list(dd_obj):
return [
{
"dataflow_id": dd_obj.get("id"),
"datafow_name": dd_obj.get("name"),
"action_type": action.get("type"),
}
for action in dd_obj.get("actions")
]
get_action_list(dd_obj=dataflow_definition_ls[0])[{'dataflow_id': 112,
'datafow_name': 'Time Zone Test',
'action_type': 'LoadFromVault'},
{'dataflow_id': 112,
'datafow_name': 'Time Zone Test',
'action_type': 'ExpressionEvaluator'},
{'dataflow_id': 112,
'datafow_name': 'Time Zone Test',
'action_type': 'DateCalculator'},
{'dataflow_id': 112,
'datafow_name': 'Time Zone Test',
'action_type': 'ExpressionEvaluator'},
{'dataflow_id': 112,
'datafow_name': 'Time Zone Test',
'action_type': 'PublishToVault'}]action_ls = [get_action_list(dd_obj) for dd_obj in dataflow_definition_ls]
flat_action_ls = [action for a_list in action_ls for action in a_list]
flat_action_ls[0:5][{'dataflow_id': 112,
'datafow_name': 'Time Zone Test',
'action_type': 'LoadFromVault'},
{'dataflow_id': 112,
'datafow_name': 'Time Zone Test',
'action_type': 'ExpressionEvaluator'},
{'dataflow_id': 112,
'datafow_name': 'Time Zone Test',
'action_type': 'DateCalculator'},
{'dataflow_id': 112,
'datafow_name': 'Time Zone Test',
'action_type': 'ExpressionEvaluator'},
{'dataflow_id': 112,
'datafow_name': 'Time Zone Test',
'action_type': 'PublishToVault'}]import pandas as pd
df = pd.DataFrame(flat_action_ls)
df[0:5]| dataflow_id | datafow_name | action_type | |
|---|---|---|---|
| 0 | 112 | Time Zone Test | LoadFromVault |
| 1 | 112 | Time Zone Test | ExpressionEvaluator |
| 2 | 112 | Time Zone Test | DateCalculator |
| 3 | 112 | Time Zone Test | ExpressionEvaluator |
| 4 | 112 | Time Zone Test | PublishToVault |
# df.to_csv("marks_csv.csv")